﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace HTTPAgilitySpider
{
    public class CrawlerSpider
    {
        public static void Crawler(string url)
        {
            List<SpiderModel> list = new List<SpiderModel>();

            try
            {
                // 1. 下载页面，获取页面数据
                string html = HttpHelper.DownloadHtml(url, Encoding.UTF8);

                // 2. 创建HTML文档对象
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(html);//加载html

                // 3. 将提出出来的数据，转换为NodelCollection集合
                string divPath = "//*[@id='plist']/ul/li"; // 数据集合xpath地址
                HtmlNodeCollection noneNodeList = doc.DocumentNode.SelectNodes(divPath);

                if (noneNodeList == null || noneNodeList.Count == 0)
                {
                    Console.WriteLine("集合为空，无数据可提取");
                    return;
                }

                // 4. 遍历集合，提取出职位信息
                foreach (var node in noneNodeList)
                {
                    HtmlDocument docChild = new HtmlDocument();
                    docChild.LoadHtml(node.OuterHtml);

                    SpiderModel sm = new SpiderModel();

                    string urls = "//*[@class='p-name']/a";
                    HtmlNode urlsNode = docChild.DocumentNode.SelectSingleNode(urls);
                    if (urlsNode == null)
                    {
                        continue;
                    }
                    sm.Url = urlsNode.InnerHtml;


                    string titlePath = "//*[@class='p-name']/a/em";
                    HtmlNode titlePathNode = docChild.DocumentNode.SelectSingleNode(titlePath);
                    if (titlePathNode == null)
                    {
                        continue;
                    }
                    sm.Title = titlePathNode.InnerHtml;


                    string pricePath = "//*[@class='p-price']/strong/i";
                    HtmlNode pricePathNode = docChild.DocumentNode.SelectSingleNode(pricePath);
                    if (pricePathNode == null)
                    {
                        continue;
                    }
                    sm.Price = pricePathNode.InnerHtml;




                    list.Add(sm);
                    Console.WriteLine();
                    Console.WriteLine();
                }

                Console.WriteLine($"一共获取到了:{list.Count()}条数据！");
            }
            catch (Exception ex)
            {
                // 写入日志
                Console.WriteLine(ex.Message);
            }
        }
    }
}
